import pandas as pd
from sqlalchemy import create_engine

engine = create_engine(
    'mysql+pymysql://root:123456@127.0.0.1:3306/test?charset=utf8')
sql = pd.read_sql('all_gzdata', engine, chunksize=10000)

for i in sql:
    d = i[['realIP', 'fullURL']]  # 只要网址列
    d = d[d['fullURL'].str.contains('\.html')].copy()  # 只要含有.html的网址
    # 保存到数据库的cleaned_gzdata表中（如果表不存在则自动创建）
    d.to_sql('cleaned_gzdata', engine, index=False, if_exists='append')
